#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@File    :   as_number.py    
@Contact :   291622538@qq.com

@Modify Time      @Author    @Version    @Description
------------      -------    --------    -----------
2020/10/23 21:35   fan        1.0         爬取自治系统号码数据
'''
from scrapy import Spider, Request
from website.items import *


class AsNumber(Spider):
    # 爬虫名称
    name = "as_number_spider"
    # 专用的通道
    custom_settings = {
        'ITEM_PIPELINES': {'website.pipelines.SaveAsNumberPipeline': 600},
    }

    # 爬虫入口
    def start_requests(self):

        URL = "http://as.chacuo.net/list"
        # dont_filter 相同的url只请求一次： False
        yield Request(URL, callback=self.as_number_list_parse, dont_filter=False)

    def as_number_list_parse(self, response, **kwargs):
        as_num_tr = response.xpath('//*[@id="main"]/div[3]/div/table/tbody/tr[@class="even" or @class="odd"]')
        for as_num in as_num_tr:
            ranking = "".join(as_num.xpath('td[1]/text()').extract())
            twoCode = "".join(as_num.xpath('td[2]/a/text()').extract())
            countryName = "".join(as_num.xpath('td[3]//a/text()').extract())
            countryHref = "".join(as_num.xpath('td[3]//a/@href').extract())
            asSum = int("".join(as_num.xpath('td[4]/text()').extract()).replace(",", ""))

            item = AsListItem(ranking=ranking, twoCode=twoCode, countryName=countryName, countryHref=countryHref,
                              asSum=asSum)
            yield Request(countryHref, meta={"item": item}, callback=self.country_as_list_parse, dont_filter=False)

    def country_as_list_parse(self, response, **kwargs):
        country_as_tr = response.xpath('//*[@id="main"]/div[3]/div/table/tbody/tr[@class="even" or @class="odd"]')
        for country_as in country_as_tr:
            asRanking = "".join(country_as.xpath('td[1]/text()').extract())
            asNum = "".join(country_as.xpath('td[2]//a/text()').extract())
            asNumHref = "".join(country_as.xpath('td[2]//a/@href').extract())
            registeredInstitution = "".join(country_as.xpath('td[3]/text()').extract())
            ipNum = int("".join(country_as.xpath('td[4]/text()').extract()).replace("个", "").replace(",", ""))
            registrationTime = "".join(country_as.xpath('td[5]/text()').extract())
            ipSegmentHref = "http://as.chacuo.net/down/t_txt=c_%s" % asNum.replace(" AS", "")
            item = response.meta["item"]
            item["asRanking"] = asRanking
            item["asNum"] = asNum
            item["registeredInstitution"] = registeredInstitution
            item["ipNum"] = ipNum
            item["registrationTime"] = registrationTime
            item["asNumHref"] = asNumHref
            item["ipSegmentHref"] = ipSegmentHref

            yield Request(ipSegmentHref, meta={"item": item}, callback=self.as_ip_list_parse, dont_filter=False)

    def as_ip_list_parse(self, response, **kwargs):

        ipSegment = "".join(response.xpath('/html/body/pre/text()').extract())
        print(ipSegment)
        item = response.meta["item"]
        item["ipSegment"] = ipSegment
        print(item)
        yield item
